package fetcher

import (
	"bufio"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"io/ioutil"
	"learngo/crawler/config"
	"log"
	"net/http"
	"time"
)

var (
	rateLimiter = time.Tick(
		time.Second / config.Qps)
	verboseLogging = false
)
func Fetcher(url string) ([]byte, error){
	//限速
	<-rateLimiter
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	contentType := res.Header.Get("Content-Type") // Optional, better guessing
	utf8reader, err := charset.NewReader(res.Body, contentType)
	if err != nil {
		return nil, err
	}
	return ioutil.ReadAll(utf8reader)
}

func determineEncoding(r *bufio.Reader) (encoding.Encoding,string)  {
	bytes, err := r.Peek(1024)
	if err !=nil {
		log.Printf("Fetch encoding error: %v", err)
		return unicode.UTF8,""
	}
	e, name, _ := charset.DetermineEncoding(bytes, "")
	return e,name
}

func SetVerboseLogging() {
	verboseLogging = true
}